


def create_vocab(data_list,thresh=5):
    ###  count occurences ###
    word2count = {}

    for data in data_list:
        for line in data:
            for word in line.split():
                if word not in word2count:
                    word2count[word] = 1
                else:
                    word2count[word] += 1

    vocab = {}
    tokens = ['<PAD>', '<EOS>', '<OUT>', '<SOS>']
    x = len(vocab)
    for token in tokens:
        vocab[token] = x
        x += 1

    for word, count in word2count.items():
        if count >= thresh:
            vocab[word] = x
            x += 1
    return vocab

def convert_sentence_to_ids(sen,vocab):
    lst = []
    for word in sen.split():
        if word not in vocab:
            lst.append(vocab['<OUT>'])
        else:
            lst.append(vocab[word])
    return lst

